import json, os
def to_normal(dic):
    assert isinstance(dic, dict)
    convs = dic['conversations']
    values = [conv['value'] for conv in convs]
    text = '\n'.join(values)
    return {'id': dic['id'], 'text': text}

def parse(filepath):
    fileopath = os.path.join(os.path.dirname(filepath), 'norm-'+os.path.basename(filepath))
    with open(filepath, 'r') as f, open(fileopath, 'w') as fo:
        for idx, line in enumerate(f):
            dic = json.loads(line)
            fo.write(json.dumps(to_normal(dic), ensure_ascii=False)+'\n')
            if idx % 10000 ==0:
                print(idx)

parse('/share/projset/dsir7/7M_0712_math_plus_system_release_0802.jsonl')
parse('/share/projset/dsir7/0729_filter_release.jsonl')
